{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3f84024c-37dc-435a-97fb-441fadb97132",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from utils.preprocessing import transtime,tracelog,servicelog,read_filter_csv,read_apm_pod,read_apm_service\n",
    "import json\n",
    "import re\n",
    "import datetime\n",
    "from utils.analysis import *\n",
    "from utils.prompt import *\n",
    "from utils.llm import *\n",
    "from tqdm import tqdm\n",
    "from openai import OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a4d86128-b69a-4468-9705-8f98816c000d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I'm just a computer program, so I don't have feelings, but I'm here and ready to help you with anything you need! How about you—how are you doing today? 😊\n"
     ]
    }
   ],
   "source": [
    "#client = OpenAI(api_key=\"79ab5d115bf2b7d736978b1e1d6232071361ae27656de07ab86c8e3c3a5602e9\", base_url=\"https://uni-api.cstcloud.cn/v1\")\n",
    "client = OpenAI(api_key=\"sk-5c5340b724ff48199ced5b4e4779123b\", base_url=\"https://api.deepseek.com\")\n",
    "callllm = llm_handler(client)\n",
    "print(callllm('how are you'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c580432e-071d-40ee-9c34-6d9c1354058f",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = json.load(open('/app/jupyter/aiops/phaseone/input.json','r'))\n",
    "basepath = '/app/jupyter/aiops/phaseone/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "772b9240-ad11-4845-a682-74cff53c1d62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start from 104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/107 [02:02<3:36:49, 122.73s/it]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 33\u001b[0m\n\u001b[1;32m     30\u001b[0m basepathday \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mbasepath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdays[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m     32\u001b[0m \u001b[38;5;66;03m# Read in trace first\u001b[39;00m\n\u001b[0;32m---> 33\u001b[0m trace \u001b[38;5;241m=\u001b[39m \u001b[43mtracelog\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbasepath\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mbasepathday\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m/trace-parquet/\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43mdatetime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mhours\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43mfromt\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdatetimes\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtot\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdatetimes\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mpadding\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     34\u001b[0m trace\u001b[38;5;241m.\u001b[39mpreprocess()\n\u001b[1;32m     35\u001b[0m tracedata \u001b[38;5;241m=\u001b[39m trace\u001b[38;5;241m.\u001b[39mdata\n",
      "File \u001b[0;32m/app/jupyter/aiops/channel1/utils/preprocessing.py:205\u001b[0m, in \u001b[0;36mtracelog.__init__\u001b[0;34m(self, basepath, datetime, fromt, tot)\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    204\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 205\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata \u001b[38;5;241m=\u001b[39m \u001b[43mload_in_file\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mbasepath\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m/trace_jaeger-span_\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43midatetime\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m.parquet\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    206\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m    207\u001b[0m         \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mFailed to loadding \u001b[39m\u001b[38;5;132;01m{\u001b[39;00midatetime\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[0;32m/app/jupyter/aiops/channel1/utils/load_data.py:5\u001b[0m, in \u001b[0;36mload_in_file\u001b[0;34m(filename)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_in_file\u001b[39m(filename):\n\u001b[1;32m      4\u001b[0m     parquet_file \u001b[38;5;241m=\u001b[39m pq\u001b[38;5;241m.\u001b[39mParquetFile(filename)\n\u001b[0;32m----> 5\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[43mparquet_file\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_pandas\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      6\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m data\n",
      "File \u001b[0;32m/var/lib/docker/anaconda3/envs/aiops/lib/python3.10/site-packages/pyarrow/array.pxi:885\u001b[0m, in \u001b[0;36mpyarrow.lib._PandasConvertible.to_pandas\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m/var/lib/docker/anaconda3/envs/aiops/lib/python3.10/site-packages/pyarrow/table.pxi:5002\u001b[0m, in \u001b[0;36mpyarrow.lib.Table._to_pandas\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m/var/lib/docker/anaconda3/envs/aiops/lib/python3.10/site-packages/pyarrow/pandas_compat.py:784\u001b[0m, in \u001b[0;36mtable_to_dataframe\u001b[0;34m(options, table, categories, ignore_metadata, types_mapper)\u001b[0m\n\u001b[1;32m    781\u001b[0m columns \u001b[38;5;241m=\u001b[39m _deserialize_column_index(table, all_columns, column_indexes)\n\u001b[1;32m    783\u001b[0m column_names \u001b[38;5;241m=\u001b[39m table\u001b[38;5;241m.\u001b[39mcolumn_names\n\u001b[0;32m--> 784\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mpa\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtable_to_blocks\u001b[49m\u001b[43m(\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtable\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcategories\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    785\u001b[0m \u001b[43m                                \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mext_columns_dtypes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkeys\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    786\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m _pandas_api\u001b[38;5;241m.\u001b[39mis_ge_v3():\n\u001b[1;32m    787\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapi\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01minternals\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m create_dataframe_from_blocks\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "padding = 1000\n",
    "full_result = []\n",
    "curpoint = 0\n",
    "try:\n",
    "    with open('result.jsonl','r') as fp:\n",
    "        curpoint = len(fp.readlines())\n",
    "except Exception as err:\n",
    "    pass\n",
    "\n",
    "print(f'Start from {curpoint}')\n",
    "\n",
    "for ano in tqdm(inputs[curpoint:]):\n",
    "    history = ''\n",
    "    error_service_explained = []\n",
    "    error_node_explained = []\n",
    "    error_services = []\n",
    "    error_nodes = []\n",
    "    des = ano['Anomaly Description']\n",
    "    uuid = ano['uuid']\n",
    "    pattern=re.compile('2025-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z')  \n",
    "    result=pattern.findall(des)\n",
    "    #print(result)\n",
    "    result_8 = [(datetime.datetime.strptime(i, \"%Y-%m-%dT%H:%M:%SZ\")+ datetime.timedelta(hours=8)).strftime(\"%Y-%m-%dT%H:%M:%SZ\") for i in result]\n",
    "    #print(result_8)\n",
    "    datetimes = [transtime(i) for i in result ] \n",
    "    hours = [i[0:13].replace('T','_')+'-00-00' for i in result_8]\n",
    "    days = [i[0:10] for i in result_8]\n",
    "\n",
    "    # basepathday\n",
    "    basepathday = f'{basepath}/{days[0]}'\n",
    "    \n",
    "    # Read in trace first\n",
    "    trace = tracelog(basepath=f'{basepathday}/trace-parquet/',datetime=[hours[0]],fromt = datetimes[0]-padding,tot = datetimes[1] + padding)\n",
    "    trace.preprocess()\n",
    "    tracedata = trace.data\n",
    "    # read in log then\n",
    "    log = servicelog (basepath=f'{basepathday}/log-parquet/',datetime=[hours[0]],fromt = datetimes[0]-padding,tot = datetimes[1] + padding)\n",
    "    log.preprocess()\n",
    "    logdata = log.fulllogs\n",
    "    # Read in infra info\n",
    "    nodeinfradata = read_filter_csv(path=f'{basepathday}/metric-parquet/infra/node_all.csv',fromt = datetimes[0]-padding,tot = datetimes[1]+ padding)\n",
    "    # Read in pod info\n",
    "    podinfradata = read_filter_csv(path=f'{basepathday}/metric-parquet/infra/pod_all.csv',fromt = datetimes[0]-padding,tot = datetimes[1]+ padding)\n",
    "\n",
    "    success = False\n",
    "\n",
    "    history = ''\n",
    "    context = ''\n",
    "    \n",
    "    while True:\n",
    "        history,suffix = prompt(des,history)\n",
    "        actions = callllm('\\n'.join([history,suffix]))\n",
    "        #print('\\n************************ROUND*********************\\n')\n",
    "        #print(f'\\n {actions}')\n",
    "        history += f'\\n## NEW ROUND\\n - THINKING: \\n {actions}'\n",
    "        function = ''\n",
    "        for i in actions.split('\\n'):\n",
    "            if 'METHOD:' in i:\n",
    "                function = i.split(':')[1].strip()\n",
    "            elif 'SERVICE:' in i:\n",
    "                error_service_explained.append(i.split(':')[1])\n",
    "                error_services.append(i.split(':')[1].split(':')[0].strip()) \n",
    "            elif 'NODE:' in i:\n",
    "                error_node_explained.append(i.split(':')[1])\n",
    "                error_nodes.append(i.split(':')[1].split(':')[0].strip())\n",
    "            elif 'GOCHA:' in i:\n",
    "                success = True\n",
    "            else:\n",
    "                continue\n",
    "\n",
    "        if success:\n",
    "            break\n",
    "     \n",
    "        if function !='':\n",
    "            if 'LogSearch' in function:\n",
    "                result = LogSearch(logdata)\n",
    "            elif 'TraceAnalysis' in function :\n",
    "                result,error_services_tmp = TraceAnalysis(tracedata)\n",
    "                if len(error_services_tmp) >0:\n",
    "                    error_services += error_services_tmp\n",
    "            elif 'LoadMetrics_service' in function:\n",
    "                result = LoadMetrics_service(basepath = basepathday+'/metric-parquet/apm/service/',date = days[0],pod=function.split('(')[1].split(')')[0],\n",
    "                                      fromt = datetimes[0]-padding,tot = datetimes[1]+ padding,padding = padding)\n",
    "            elif 'LoadMetrics_pod' in function:\n",
    "                result,service_nodes =  LoadMetrics_pod(podinfradata,nodeinfradata,error_services = [function.split('(')[1].split(')')[0]],\n",
    "                                                               fromt = datetimes[0]-padding,tot = datetimes[1]+ padding,padding = padding,\n",
    "                                                               basepathday = basepathday,days =  days)\n",
    "            else: \n",
    "                break\n",
    "        \n",
    "        history += f'\\n - ACTION:called function {function} '\n",
    "        history += f'\\n - RESULT: {result} '\n",
    "\n",
    "        #print('HISTORY:------------------------\\n',history)\n",
    "        \n",
    "    #print('\\n++++++++++++++++++Full history+++++++++++++++++\\n')\n",
    "    #print(history)\n",
    "    #break\n",
    "    result = callllm(gen_result(history,uuid))\n",
    "    if result.strip()[0] == '`':\n",
    "        result = result[7:-3]\n",
    "    with open('result.jsonl','a') as fp:\n",
    "        fp.write(json.dumps(json.loads(result))+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95dcb790-9120-4ec9-9029-97ccb8255ee9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
